package com.spider.moudle.crawler.forum;


import com.alibaba.fastjson.JSONObject;
import com.spider.moudle.api.Crawler;
import com.spider.moudle.api.LoginDownloader;
import com.spider.moudle.api.realize.AbstractParamCrawler;
import com.spider.moudle.api.realize.AbstractReplyCrawler;
import com.spider.moudle.browser.CommonDriverUtil;
import com.spider.moudle.entity.Area;
import com.spider.moudle.entity.Cookie;
import com.spider.moudle.entity.ResponseEntity;
import com.spider.moudle.entity.Setting;
import com.spider.moudle.entity.SpeechBean;
import com.spider.moudle.entity.Task;
import com.spider.moudle.entity.TaskType;
import com.spider.moudle.entity.UserBean;
import com.spider.moudle.entity.WebType;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.SettingsUtil;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.WebDriver;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;

public class ZuiChengDuCrawler extends AbstractParamCrawler {
    private ArrayList<Cookie> cookies;
    private LoginDownloader mDownloader;
    // TODO: 2017-12-21 0021 测试
    private static ZuiChengDuCrawler mZuiChengDuCrawler;
    /**
     * 这个是每个网站的个性化配置，是从settings.json里面读取到的(必须)
     */
    private final Setting config = SettingsUtil.getSettingBySite("ZuiChengDu");


    @Override
    protected ZuiChengDuCrawler clone() {
        return new ZuiChengDuCrawler(this.cookies, this.mDownloader);
    }

    @Override
    public LinkedList<Crawler> initial() {
        LinkedList<Crawler> list = new LinkedList<>();
        int count = config.getCrawler_num();
        for (int i = 0; i < count; i++) {
            list.add(this.clone());
        }
        return list;
    }

    public ZuiChengDuCrawler(ArrayList<Cookie> cookies, LoginDownloader downloader) {
        this.cookies = cookies;
        mDownloader = downloader;
    }


    public ZuiChengDuCrawler() {
        WebDriver driver = null;
        try {
            driver = CommonDriverUtil.getDriver();
            driver.get(config.getIndex());
            try {
                Thread.sleep(6000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            cookies = Cookie.tranlateToCookie(driver.manage().getCookies());
            mDownloader = DownloaderFactory.buildLoginDownloader(cookies, "GBK");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            driver.quit();
        }
    }


    @Override
    public void generateTask() {
        ExecutorService executorService = null;
        try {
            ResponseEntity entity = mDownloader.get(config.getIndex());
            if (entity.getStatus_code() == 200) {
                Document document = entity.getDocument();
                Elements elements = document.select("div[class=\"bm bmw  flg cl\"] dl");
                if (elements.isEmpty()) {
                    return;
                }
                executorService = ExecutorServiceFactoy.buildExecutorService(config.getTask_thread_count(),
                        ExecutorServiceFactoy.getName(config.getName(), config.getIndex()), new NamedThreadFactory
                                (config.getName(), "生成任务"));
                for (Element element : elements) {
                    Element node = element.select("a").get(0);
                    String url = node.attr("href");
                    GetPostTask getPostTask = new GetPostTask(node.text(), url);
                    executorService.submit(getPostTask);
                }
                // TODO: 2017-12-18 0018 测试代码
                //                Element testElement = elements.get(1);
                //                if (testElement != null) {
                //                    Element node = testElement.select("a").get(0);
                //                    String url = node.attr("href");
                //                    GetPostTask getPostTask = new GetPostTask(node.text(), url);
                //                    executorService.submit(getPostTask);
                //                }

            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (executorService != null) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        JSONObject data = new JSONObject();
        switch (task.getTaskType()) {
            case SPEECH:
                SpeechBean speechBean = new SpeechBean();
                getSpeech(task.getUrl(), task.getArea(), speechBean, true);
                data.put("data", speechBean.toJSONObject());
                break;
            case USER:
                UserBean userBean = new UserBean();
                getUserInfo(task, userBean);
                //System.out.println(GsonUtils.toJson(userBean));
                data.put("data", userBean.toJSONObject());
                break;
            default:
                data = null;
                break;
        }
        //System.out.println(data);
        return data;
    }

    /**
     * 获取个人信息
     *
     * @param task task
     * @return
     */
    private void getUserInfo(Task task, UserBean userBean) throws Exception {
        String userKey = task.getTaskKey();

        if (CommonUtil.isNull(userKey)) {
            System.err.println("User key is not empty where task type belong to home page");
            return;
        }
        String uid = userKey.split("_")[1];
        String userUrl = getUserInfoUrl(uid);

        ResponseEntity entity = mDownloader.get(userUrl);
        if (entity.getStatus_code() == 200) {
            Document document = entity.getDocument();

            //当前页面列表数据
            Elements elements = document.select("div.ct2.wp.cl");
            if (elements.isEmpty()) {
                return;
            }
            userBean.setJson_id(userKey);
            userBean.setUrl(userUrl);
            userBean.setPublish_user_id(uid);
            Elements headHead = elements.select("div.hm");

            String publish_user_photo = headHead.select("p > a > img").attr("src");
            userBean.setPublish_user_photo(publish_user_photo);

            String publish_user = headHead.select("h2.xs2 a").text();
            userBean.setPublish_user(publish_user);

            Elements liNodes = elements.select("ul.cl.bbda.pbm.mbm > li a");
            if (liNodes.size() == 5) {
                int friend_count = Integer.valueOf(liNodes.get(0).text().split("好友数 ")[1]);
                int article_count = Integer.valueOf(liNodes.get(4).text().split("主题数 ")[1]);
                int reply_article_count = Integer.valueOf(liNodes.get(3).text().split("回帖数 ")[1]);
                userBean.setFriend_count(friend_count);
                userBean.setArticle_count(article_count);
                userBean.setReply_article_count(reply_article_count);

            }


            //            elements.select("ul[class='pf_l cl'] li");
            Elements nodes = elements.select("li");
            for (Element element : nodes) {
                String key = element.select("em").text();
                element.select("em").remove();
                String value = element.text();
                switch (key.trim()) {
                    case "性别":
                        userBean.setSex(value);
                        break;
                    case "注册时间":
                        userBean.setRegister_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, value + ":00"));
                        break;
                    case "空间访问量":
                        userBean.setVisit_count(Integer.valueOf(value));
                        break;
                    case "用户组":
                        value = element.select("a").text();
                        userBean.setGroup_name(value);
                        break;
                    case "交友目的":
                        userBean.setIntroduction(value);
                        break;
                    default:
                        //                        System.out.println("|" + key.trim() + "|");
                        break;
                }
            }
        }
    }

    /**
     * 获取言论
     *
     * @param url         postUrl
     * @param area        area
     * @param speechBean  speechBean
     * @param isFirstPage 是否是第一页
     * @throws Exception Exception
     */
    private void getSpeech(String url, Area area, SpeechBean speechBean, boolean isFirstPage) throws Exception {

        List<SpeechBean> reply_nodes = speechBean.getReply_nodes();

        ResponseEntity entity = mDownloader.get(url);
        if (entity.getStatus_code() == 200) {
            Document document = entity.getDocument();
            //当前页面列表数据
            Elements elements = document.select("div#postlist.pl");
            if (elements.isEmpty()) {
                return;
            }
            String tid = "";
            Elements postList = elements.select("div.rtv");

            if (isFirstPage) {

                getSpeechItem(url, postList.get(0), speechBean, "", "");

                speechBean.setSpider_time(new Date());

                tid = url.split("&")[1].split("=")[1];
                speechBean.setId(tid);

                String title = document.select("div.post_title h1 a#thread_subject").text();
                speechBean.setTitle(title);
                String likeCount = document.select("div.user_btn a.give").text();
                if (CommonUtil.isNotNull(likeCount)) {
                    int like_count = Integer.valueOf(likeCount);
                    speechBean.setLike_count(like_count);
                }

                speechBean.setUrl(url);
                speechBean.setArea(area);
                Element infoElement = elements.select("div.posts_info.mtp").get(0);
                String[] read_replay = infoElement.select("span.txt.rf").get(0).text().split(" ");
                //阅读量
                int readCount = Integer.valueOf(read_replay[0].split(":")[1]);
                speechBean.setRead_count(readCount);
                //回复数
                int replayCount = Integer.valueOf(read_replay[1].split(":")[1]);
                speechBean.setReply_count(replayCount);
                //移除帖子内容
                postList.remove(0);
            }

            for (Element post : postList) {
                if (post.select("td.t_f").size() != 0) {//有内容的评论
                    SpeechBean reply = new SpeechBean();
                    getSpeechItem(url, post, reply, tid, tid);
                    reply.setArea(area);
                    reply_nodes.add(reply);
                }
            }

            String pageUrl = document.select("#pgt > div > div > a.nxt").attr("href");
            if (CommonUtil.isNotNull(pageUrl)) {
                String baseUrl = pageUrl.substring(0, pageUrl.lastIndexOf("=") + 1);
                String pageTitle = document.select("#pgt > div > div > label > span").attr("title");
                int pageNum = Integer.valueOf(pageTitle.substring(1, pageTitle.length() - 1).trim());
                //爬虫最大页码的限制，超过最大页的帖子就只抓到最大页码数
                if (pageNum >= config.getMax_page_count()) {
                    pageNum = config.getMax_page_count();
                }
                int threadNum = config.getReply_thread_count();
                ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, pageNum, threadNum);

                ExecutorService executorService = null;
                List<FutureTask<List<SpeechBean>>> futureTasks = new ArrayList<>();
                try {
                    executorService = ExecutorServiceFactoy.buildExecutorService(splitPages.size(),
                            ExecutorServiceFactoy.getName(config.getName(), url), new NamedThreadFactory(config
                                    .getName(), "回复"));

                    for (int i = 0; i < splitPages.size(); i++) {
                        ReplyCrawler callable = new ReplyCrawler(splitPages.get(i), baseUrl, tid, tid);
                        FutureTask<List<SpeechBean>> futureTask = new FutureTask<>(callable);
                        futureTasks.add(futureTask);
                        executorService.submit(futureTask);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error("最成都获取回复线程执行异常");
                } finally {
                    if (null != executorService) {
                        executorService.shutdown();
                    }
                }

                for (FutureTask<List<SpeechBean>> futureTask : futureTasks) {
                    reply_nodes.addAll(futureTask.get());
                }
            }

            Collections.sort(reply_nodes, (o1, o2) -> {
                Date s1 = o1.getPublish_time();
                Date s2 = o2.getPublish_time();
                int flag = s1.compareTo(s2);
                return flag;
            });
        } else {
            System.out.println(entity.getStatus_code());
        }
    }

    /**
     * 获取每个item内容
     *
     * @param post 每个item
     * @return SpeechBean
     * @throws ParseException ParseException
     */
    private void getSpeechItem(String postUrl, Element post, SpeechBean reply, String ancestor_id, String parent_id)
            throws Exception {
        Element publish_user_node = post.select("a.xw1").get(0);
        //回复的ID
        String id = post.select("div.rtv table").attr("id").substring(3);
        reply.setId(id);
        reply.setUrl(postUrl + "#pid" + id);

        reply.setAncestor_id(ancestor_id);
        reply.setParent_id(parent_id);

        String publish_user = publish_user_node.text();
        reply.setPublish_user(publish_user);

        String publish_user_id = publish_user_node.attr("href").split("&")[1].split("=")[1];
        reply.setPublish_user_id(publish_user_id);

        reply.setPublish_user_jsonid(config.getName() + "_" + publish_user_id);
        //        testUser(config.getName() + "_" + publish_user_id);
        // TODO: 2017-12-29 0029 测试时注释
        addUserToTask(config.getName() + "_" + publish_user_id);

        Element timeNode = post.select("em[id^='authorposton']").get(0);
        String time;
        if (timeNode.select("span").size() == 0) {
            time = timeNode.text().substring(4);
        } else {
            time = timeNode.select("span").get(0).attr("title");
        }

        reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, time));

        String publish_user_photo = post.select("div.avatar > a > img").get(0).attr("src");
        reply.setPublish_user_photo(publish_user_photo);

        Element contentNode;
        if (post.select("div.t_fsz").size() != 0) {//普通帖子
            contentNode = post.select("div.t_fsz").get(0);
        } else if (post.select("div.pcbs").size() != 0) {//投票贴
            contentNode = post.select("div.pcbs").get(0);
        } else {
            contentNode = null;
            System.out.println("暂无数据解析：" + postUrl);
        }
        if (contentNode != null) {
            contentNode.select("div i").remove();//移除本贴由谁最后编辑
            contentNode.select("div.quote").remove();//回复某一个帖子
            contentNode.select("i").remove();//移除该帖不存在
            contentNode.select("div[class^='tip ']").remove();//图片下载内容
            contentNode.select("p.mbn").remove();//图片下载内容

            Elements imgs = contentNode.select("img");
            List<String> imgUrls = reply.getImg_urls();
            for (Element imgNode : imgs) {
                String halfUrl = imgNode.attr("src");
                if (halfUrl.endsWith("none.gif")) {//未加载出来的图片数据
                    halfUrl = imgNode.attr("zoomfile");
                }
                if (CommonUtil.isNull(halfUrl)) {//src不存在的情况
                    halfUrl = imgNode.attr("file");
                }

                if (!halfUrl.startsWith("http") && !halfUrl.startsWith("https")) {
                    String url;
                    url = "http://www.chengtu.com/" + halfUrl;
                    imgUrls.add(url);
                } else {
                    imgUrls.add(halfUrl);
                }
            }
            reply.setImg_urls(imgUrls);
            String content;

            content = contentNode.text();
            reply.setContent(content);
        }
    }

    /**
     * 获取板块内帖子URL
     */
    private class GetPostTask implements Runnable {
        private String mUrl;
        private Area mArea = Area.search("成都市");
        private String mForumName;
        private boolean isToday = false;

        public GetPostTask(String forumName, String url) {
            mUrl = url;
            mForumName = forumName;
            //            System.out.println(forumName + ":" + mUrl);
        }

        @Override
        public void run() {
            ExecutorService executorService = null;
            String url = "";
            try {
                ResponseEntity entity = mDownloader.get(mUrl);
                if (entity.getStatus_code() == 200) {
                    Document document = entity.getDocument();

                    Elements dataElements = document.select("div.bm_c table tbody[id^='normalthread_']");
                    for (Element element : dataElements) {
                        Element timeNode = element.select("td:nth-child(5) em").get(0);
                        Element titleNode = element.select(".xst").get(0);

                        String title = titleNode.text();
                        url = titleNode.attr("href");
                        String time = timeNode.text();
                        if (timeNode.select("span").size() != 0) {
                            time = timeNode.select("span").attr("title");
                        }
                        if (DateUtil.isToday(time, DateUtil.yyyy_MM_dd_HH_mm)) {
                            isToday = true;
                            //                            System.out.println("【" + mForumName + "】：" + title + "：" +
                            // url + "：" + time);
                            String key = config.getSite() + "_" + url.split("&")[1].split("=")[1];
                            //                            testSpeech(url, key);
                            // TODO: 2017-12-26 0026 测试时注销
                            addUrlToTask(url, key, mArea);
                        } else {
                            isToday = false;
                        }
                    }
                    String nextUrl = document.select(".nxt").attr("href");
                    if (!CommonUtil.isNull(nextUrl) && isToday) {
                        executorService = ExecutorServiceFactoy.buildExecutorService(1, ExecutorServiceFactoy.getName
                                (config.getName(), config.getIndex()), new NamedThreadFactory(config.getName(),
                                "生成任务"));
                        GetPostTask getPostTask = new GetPostTask(mForumName, nextUrl);
                        executorService.submit(getPostTask);
                    }

                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (executorService != null) {
                    executorService.shutdown();
                    while (!executorService.isTerminated()) {
                        try {
                            Thread.sleep(1500);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        }

    }

    /**
     * 获取回复
     */
    private class ReplyCrawler extends AbstractReplyCrawler {
        private ArrayList<Integer> pages;

        public ReplyCrawler(ArrayList<Integer> pages, String baseUrl, String parentId, String ancestorId) {
            this.pages = pages;
            this.url = baseUrl;
            this.parentId = parentId;
            this.ancestorId = ancestorId;
        }

        @Override
        public List<SpeechBean> call() throws Exception {
            List<SpeechBean> replyList = new ArrayList<>();

            for (Integer page : pages) {
                String replyUrl = this.url + page;
                ResponseEntity entity = mDownloader.get(replyUrl);
                if (entity.getStatus_code() == 200) {
                    Document document = entity.getDocument();
                    //当前页面列表数据
                    Elements elements = document.select("div#postlist.pl");
                    if (elements.isEmpty()) {
                        continue;
                    }

                    Elements postList = elements.select("div.rtv");
                    for (Element post : postList) {
                        if (post.select("td.t_f").size() != 0) {//有内容的评论
                            SpeechBean reply = new SpeechBean();
                            getSpeechItem(url, post, reply, this.ancestorId, this.parentId);
                            replyList.add(reply);
                        }
                    }
                }
            }
            return replyList;
        }
    }

    /**
     * 将抓取到的URL放入task
     *
     * @param url  url
     * @param key  key
     * @param area area
     * @throws Exception Exception
     */
    private void addUrlToTask(String url, String key, Area area) throws Exception {
        Task task = new Task(url, TaskType.SPEECH, WebType.FORLUM, key, area);
        TaskManager.add(config.getSpeech_task(), task);
    }

    /**
     * 将抓取到的user放入task
     *
     * @param userKey userKey
     * @throws Exception Exception
     */
    private void addUserToTask(String userKey) throws Exception {
        Task hometask = new Task("", TaskType.USER, WebType.FORLUM, userKey);
        TaskManager.add(config.getUser_task(), hometask);
    }

    /**
     * 获取个人信息主页地址
     *
     * @param uid uid
     * @return String
     */
    private String getUserInfoUrl(String uid) {
        String userUrl = "http://www.chengtu.com/home.php?mod=space&uid=" + uid + "&do=profile";
        return userUrl;
    }

    // TODO: 2017-12-20 0020 测试
    public static void main(String[] args) {
        mZuiChengDuCrawler = new ZuiChengDuCrawler();
        //        mZuiChengDuCrawler.generateTask();
        //http://www.chengtu.com/forum.php?mod=viewthread&tid=499343&extra=page%3D1   64页数据
        testSpeech("http://www.chengtu.com/forum.php?mod=viewthread&tid=545046&extra=page%3D1", "");
        //        testUser("ZuiChengDu_137");
    }

    private static void testSpeech(String url, String taskKey) {
        try {
            mZuiChengDuCrawler.crawlItem(new Task(url, TaskType.SPEECH, WebType.FORLUM, taskKey, Area.search("成都市")));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void testUser(String taskKey) {
        try {
            mZuiChengDuCrawler.crawlItem(new Task("", TaskType.USER, WebType.FORLUM, taskKey, Area.search("成都市")));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
